{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First of all we need to install [ranx](https://github.com/AmenRa/ranx)\n",
    "\n",
    "Mind that the first time you run any ranx' functions they may take a while as they must be compiled first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -U ranx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download the data we need"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import requests\n",
    "\n",
    "for file in [\"qrels\", \"run_1\", \"run_2\", \"run_3\", \"run_4\", \"run_5\"]:\n",
    "    os.makedirs(\"notebooks/data\", exist_ok=True)\n",
    "\n",
    "    with open(f\"notebooks/data/{file}.trec\", \"w\") as f:\n",
    "        master = f\"https://raw.githubusercontent.com/AmenRa/ranx/master/notebooks/data/{file}.trec\"\n",
    "        f.write(requests.get(master).text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ranx import Qrels, Run, evaluate, compare"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create Qrels and Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The standard way of creating Qrels and Run is converting Python Dictionaries\n",
    "qrels_dict = {\"q_1\": {\"d_12\": 5, \"d_25\": 3}, \"q_2\": {\"d_11\": 6, \"d_22\": 1}}\n",
    "\n",
    "run_dict = {\n",
    "    \"q_1\": {\n",
    "        \"d_12\": 0.9,\n",
    "        \"d_23\": 0.8,\n",
    "        \"d_25\": 0.7,\n",
    "        \"d_36\": 0.6,\n",
    "        \"d_32\": 0.5,\n",
    "        \"d_35\": 0.4,\n",
    "    },\n",
    "    \"q_2\": {\n",
    "        \"d_12\": 0.9,\n",
    "        \"d_11\": 0.8,\n",
    "        \"d_25\": 0.7,\n",
    "        \"d_36\": 0.6,\n",
    "        \"d_22\": 0.5,\n",
    "        \"d_35\": 0.4,\n",
    "    },\n",
    "}\n",
    "\n",
    "qrels = Qrels(qrels_dict)\n",
    "run = Run(run_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute NDCG@5\n",
    "score = evaluate(qrels, run, \"ndcg@5\")\n",
    "print(score)\n",
    "\n",
    "# Compute NDCG@3, MAP@5, and MRR\n",
    "score_dict = evaluate(qrels, run, [\"ndcg@3\", \"map@5\", \"mrr\"])\n",
    "print(score_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's load qrels and runs from files and compare them\n",
    "qrels = Qrels.from_file(\"notebooks/data/qrels.trec\", kind=\"trec\")\n",
    "\n",
    "run_1 = Run.from_file(\"notebooks/data/run_1.trec\", kind=\"trec\")\n",
    "run_2 = Run.from_file(\"notebooks/data/run_2.trec\", kind=\"trec\")\n",
    "run_3 = Run.from_file(\"notebooks/data/run_3.trec\", kind=\"trec\")\n",
    "run_4 = Run.from_file(\"notebooks/data/run_4.trec\", kind=\"trec\")\n",
    "run_5 = Run.from_file(\"notebooks/data/run_5.trec\", kind=\"trec\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compares different runs and performs statistical tests (Fisher's Randomization test)\n",
    "report = compare(\n",
    "    qrels,\n",
    "    runs=[run_1, run_2, run_3, run_4, run_5],\n",
    "    metrics=[\"map@100\", \"mrr@100\", \"ndcg@10\"],\n",
    "    max_p=0.01,  # P-value threshold\n",
    ")\n",
    "\n",
    "# The comparison results are saved in a Report instance,\n",
    "# which provides handy functionalities such as tabular formatting\n",
    "# (superscripts denote statistical significance differences)\n",
    "report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A Report can also be exported as LaTeX table ready for scientific publications\n",
    "print(report.to_latex())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "692976b4e7dc85f192e7ff7f01132f0a9cfec5b2147f9c392e0014b728af08ff"
  },
  "kernelspec": {
   "display_name": "Python 3.7.12 64-bit ('ranx': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
